﻿using System;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;

namespace DiscoveryLogic.TextMining.Tokenization
{
    public class SimpleTokenizer:ITokenizer
    {
        private const string DelimiterRegexPattern = "[,.;:!?\\(\\)\\[\\]\\+'\"\n\t\\s]";
        private Regex _Delimiter;

        public SimpleTokenizer()
        {
            this._Delimiter=new Regex(DelimiterRegexPattern,RegexOptions.IgnoreCase);
        }

        #region ITokenizer Members

        public List<string> TokenizeDocument(string content)
        {
            List<string> tokens = new List<string>();
            if(!string.IsNullOrEmpty(content))
            {
                string[] words = this._Delimiter.Split(content);
                tokens.AddRange(words);
            }
            return tokens;
        }

        #endregion
    }
}
